Add value_counts. #63
Conversation
|
cc @thunterdb |
databricks/koala/structures.py
Outdated
| raise NotImplementedError("value_counts currently does not support bins") | ||
|
|
||
| if dropna: | ||
| df_dropna = self.to_dataframe().filter(self._spark_isNotNull()) |
There was a problem hiding this comment.
is this enough to filter not only the null but also the NaNs?
There was a problem hiding this comment.
Not for this PR, but I am also marking this as something to cover in the guide in #34 .
There was a problem hiding this comment.
@thunterdb updated to check the NaNs.
databricks/koala/structures.py
Outdated
|
|
||
| if dropna: | ||
| if isinstance(self.schema[self.name].dataType, (FloatType, DoubleType)): | ||
| pred = ~(self._spark_isNull() | F._spark_isnan(self)) |
There was a problem hiding this comment.
You just implemented Series.isnull(). This is fine, we can refactor this piece later. I made an issue to do that later.
databricks/koala/structures.py
Outdated
| @@ -534,10 +560,14 @@ def dropna(self, axis=0, how='any', thresh=None, subset=None, inplace=False): | |||
| else: | |||
| columns = list(self.columns) | |||
|
|
|||
| def pred(c): | |||
There was a problem hiding this comment.
Since you are repeating this logic twice already, would you mind going the whole way and defining PandasLikeSeries.inull? We can add the tests later when fully implementing #64
|
|
||
| isnull = isna | ||
|
|
||
| def notna(self): |
There was a problem hiding this comment.
I will add a doc here when working on #64.
| @@ -260,6 +260,17 @@ def to_dataframe(self): | |||
| def toPandas(self): | |||
| return _col(self.to_dataframe().toPandas()) | |||
|
|
|||
| def isna(self): | |||
No description provided.